#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: send_online_beanstalk.py
@time: 2018/1/25 16:10
"""
import json
import sys
import time

import click

sys.path.append("..")
sys.path.append("../..")
from common import util
from common.mongo import MongDb
from config.mongo_conf import MONGO_DB_SOURCE
from config.app_conf import PROVINCE_CODE_DICT, SEARCH_LIST_HISTORY_NAME_FOUND
from logger import Logger

from common.pybeanstalk import PyBeanstalk
from common.table_manage import TableManage
from model.crawl_model import STORE_TYPE_DETAIL

beanstalk_consumer_conf = {'host': 'cs6', 'port': 11400, 'tube': 'test_list_download'}

# 日志模块
log = Logger('send_online_beanstalk.log').get_logger()

source_db = MongDb(MONGO_DB_SOURCE['host'], MONGO_DB_SOURCE['port'], MONGO_DB_SOURCE['db'],
                   MONGO_DB_SOURCE['username'],
                   MONGO_DB_SOURCE['password'], log=log)


@click.command()
@click.option('--file_path',
              default='/Users/you/project/new_gsxt_crawl/script/scheduler/online_scheduler.txt',
              help='存储消息队列')
def main(file_path):
    beanstalk = PyBeanstalk(beanstalk_consumer_conf['host'], beanstalk_consumer_conf['port'])
    tube = beanstalk_consumer_conf['tube']

    company_list = []
    with open(file_path) as p_file:
        for line in p_file:

            line = line.strip().strip('\n').strip('\r')
            province, company = line.split(' ')
            province = province.strip()
            company = company.strip()

            if province not in PROVINCE_CODE_DICT:
                log.error("当前省份错误: line = {}".format(line))
                continue

            company_list.append((province, company))

    log.info("当前需要抓取的企业数目: {}".format(len(company_list)))

    success_list = [False] * len(company_list)
    while True:

        is_scheduler = False
        for index, (province, company) in enumerate(company_list):

            if success_list[index]:
                continue

            seed_table = TableManage.get_seed_table(province)

            is_need_crawl = False
            while True:
                item = source_db.find_one(seed_table, {'_id': company})
                if item is None:
                    is_need_crawl = True
                    source_db.insert_batch_data(seed_table, [{
                        '_id': company,
                        '_in_time': util.get_now_time()
                    }])
                    break

                search_status = item.get("search_status")
                if search_status == 2:
                    log.warn("当前企业未搜索到任何结果: province = {} company = {}".format(province, company))
                    break

                # 曾用名
                if search_status == SEARCH_LIST_HISTORY_NAME_FOUND:
                    log.warn("曾用名企业: province = {} company = {}".format(province, company))
                    is_need_crawl = True
                    break

                crawl_status = item.get('crawl_status')
                if crawl_status == 1:
                    log.info("当前企业抓取成功: province = {} company = {}".format(province, company))
                    break

                is_need_crawl = True
                break

            if is_need_crawl:
                send_item = {
                    'province': province,
                    'company': company,
                    'seed_table': seed_table,
                    'field': STORE_TYPE_DETAIL
                }
                data_str = json.dumps(send_item)
                beanstalk.put(tube, data_str)
                log.info("发送数据: province = {} company = {}".format(province, company))
                is_scheduler = True
            else:
                success_list[index] = True
        if not is_scheduler:
            break

        log.info("完成一轮调度，休眠...")
        time.sleep(60)
    log.info("所有企业抓取完成...")


if __name__ == '__main__':
    main()
